Disco Diffusion
Contents
Disco Diffusion#
This is an simple way of creating compelling Disco Diffusion artworks for generative artists.
References:#
Note
Install ekorpkit package first.
Set logging level to Warning, if you don’t want to see verbose logging.
If you run this notebook in Colab, set Hardware accelerator to GPU.
!pip install ekorpkit[disco] exit()
from ekorpkit import eKonf
eKonf.setLogger("INFO")
eKonf.set_cuda(device=0)
print("version:", eKonf.__version__)
is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
eKonf.mount_google_drive(
workspace="MyDrive/colab_workspace", project="disco-imagen"
)
print("evironment varialbles:")
eKonf.print(eKonf.env().dict())
INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB']
INFO:ekorpkit.base:Google Colab not detected.
version: 0.1.36+6.ge523620.dirty
is notebook? True
is colab? False
evironment varialbles:
{'CUDA_DEVICE_ORDER': 'PCI_BUS_ID',
'CUDA_VISIBLE_DEVICES': '0',
'EKORPKIT_CONFIG_DIR': '/workspace/projects/ekorpkit-book/config',
'EKORPKIT_DATA_DIR': None,
'EKORPKIT_LOG_LEVEL': 'INFO',
'EKORPKIT_PROJECT': 'ekorpkit-book',
'EKORPKIT_WORKSPACE_ROOT': '/workspace',
'KMP_DUPLICATE_LIB_OK': 'TRUE',
'NUM_WORKERS': 230}
cfg = eKonf.compose("model/disco")
disco = eKonf.instantiate(cfg)
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
INFO:ekorpkit.base:setting environment variable CACHED_PATH_CACHE_ROOT to /workspace/.cache/cached_path
INFO:ekorpkit.base:setting environment variable KMP_DUPLICATE_LIB_OK to TRUE
INFO:ekorpkit.base:Google Colab not detected.
INFO:ekorpkit.models.art.base:> downloading models...
INFO:ekorpkit.models.disco.base:Downloading model model_256 from {'SHA': '983e3de6f95c88c81b2ca7ebb2c217933be1973b1ff058776b970f901584613a', 'filename': '256x256_diffusion_uncond.pt', 'link': 'https://openaipublic.blob.core.windows.net/diffusion/jul-2021/256x256_diffusion_uncond.pt', 'link_fb': 'https://www.dropbox.com/s/9tqnqo930mpnpcn/256x256_diffusion_uncond.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/256x256_diffusion_uncond.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/256x256_diffusion_uncond.pt'}
INFO:ekorpkit.models.disco.utils:Model model_256 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model_512 from {'SHA': '9c111ab89e214862b76e1fa6a1b3f1d329b1a88281885943d2cdbe357ad57648', 'filename': '512x512_diffusion_uncond_finetune_008100.pt', 'link': 'https://the-eye.eu/public/AI/models/512x512_diffusion_unconditional_ImageNet/512x512_diffusion_uncond_finetune_008100.pt', 'link_fb': 'https://huggingface.co/lowlevelware/512x512_diffusion_unconditional_ImageNet/resolve/main/512x512_diffusion_uncond_finetune_008100.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/512x512_diffusion_uncond_finetune_008100.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/512x512_diffusion_uncond_finetune_008100.pt'}
INFO:ekorpkit.models.disco.utils:Model model_512 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model_secondary from {'SHA': '983e3de6f95c88c81b2ca7ebb2c217933be1973b1ff058776b970f901584613a', 'filename': 'secondary_model_imagenet_2.pth', 'link': 'https://huggingface.co/spaces/huggi/secondary_model_imagenet_2.pth/resolve/main/secondary_model_imagenet_2.pth', 'link_fb': 'https://the-eye.eu/public/AI/models/v-diffusion/secondary_model_imagenet_2.pth', 'archive_path': '/workspace/data/archive/disco-imagen/models/secondary_model_imagenet_2.pth', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/secondary_model_imagenet_2.pth'}
INFO:ekorpkit.models.disco.utils:Model model_secondary already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model MiDaS from {'filename': 'dpt_large-midas-2f21e586.pt', 'link': 'https://github.com/intel-isl/DPT/releases/download/1_0/dpt_large-midas-2f21e586.pt', 'archive_path': '/workspace/data/archive/disco-imagen/models/dpt_large-midas-2f21e586.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/models/dpt_large-midas-2f21e586.pt'}
INFO:ekorpkit.models.disco.utils:Model MiDaS already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model model-lpips from {'filename': 'vgg16-397923af.pth', 'link': 'https://download.pytorch.org/models/vgg16-397923af.pth', 'archive_path': '/workspace/data/archive/disco-imagen/model-lpips/vgg16-397923af.pth', 'path': '/root/.cache/torch/hub/checkpoints/vgg16-397923af.pth'}
INFO:ekorpkit.models.disco.utils:Model model-lpips already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model AdaBins from {'filename': 'AdaBins_nyu.pt', 'link': 'https://cloudflare-ipfs.com/ipfs/Qmd2mMnDLWePKmgfS8m6ntAg4nhV5VkUyAydYBp8cWWeB7/AdaBins_nyu.pt', 'archive_path': '/workspace/data/archive/disco-imagen/pretrained/AdaBins_nyu.pt', 'path': '/workspace/projects/ekorpkit-book/disco-imagen/pretrained/AdaBins_nyu.pt'}
INFO:ekorpkit.models.disco.utils:Model AdaBins already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50 from {'filename': 'RN50.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50.pt', 'path': '/root/.cache/clip/RN50.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN101 from {'filename': 'RN101.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN101.pt', 'path': '/root/.cache/clip/RN101.pt'}
INFO:ekorpkit.models.disco.utils:Model RN101 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x4 from {'filename': 'RN50x4.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x4.pt', 'path': '/root/.cache/clip/RN50x4.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x4 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x16 from {'filename': 'RN50x16.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x16.pt', 'path': '/root/.cache/clip/RN50x16.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x16 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model RN50x64 from {'filename': 'RN50x64.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/RN50x64.pt', 'path': '/root/.cache/clip/RN50x64.pt'}
INFO:ekorpkit.models.disco.utils:Model RN50x64 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-B-32 from {'filename': 'ViT-B-32.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-B-32.pt', 'path': '/root/.cache/clip/ViT-B-32.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-B-32 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-B-16 from {'filename': 'ViT-B-16.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-B-16.pt', 'path': '/root/.cache/clip/ViT-B-16.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-B-16 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.disco.base:Downloading model ViT-L-14 from {'filename': 'ViT-L-14.pt', 'link': 'https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt', 'archive_path': '/workspace/data/archive/disco-imagen/clip/ViT-L-14.pt', 'path': '/root/.cache/clip/ViT-L-14.pt'}
INFO:ekorpkit.models.disco.utils:Model ViT-L-14 already downloaded, set check_model_SHA to true if the file is corrupt
INFO:ekorpkit.models.art.base:> loading modules...
INFO:ekorpkit.utils.lib:clip not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/CLIP as clip
INFO:ekorpkit.utils.lib:guided_diffusion.script_util not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/guided-diffusion as guided_diffusion.script_util
INFO:ekorpkit.utils.lib:resize_right not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/ResizeRight as resize_right
INFO:ekorpkit.utils.lib:py3d_tools not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/pytorch3d-lite as py3d_tools
INFO:ekorpkit.utils.lib:midas.dpt_depth not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/MiDaS as midas.dpt_depth
INFO:ekorpkit.utils.lib:utils not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/MiDaS as midas_utils
INFO:ekorpkit.utils.lib:infer not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/AdaBins as infer
INFO:ekorpkit.utils.lib:utils.flow_viz not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/RAFT/core as flow_viz
INFO:ekorpkit.utils.lib:utils.utils not imported, loading from /workspace/projects/ekorpkit-book/disco-imagen/libs/RAFT/core as utils.utils
INFO:ekorpkit.utils.lib:raft imported
INFO:ekorpkit.models.art.base:> loading models...
INFO:ekorpkit.models.disco.base:> loading diffusion models...
INFO:ekorpkit.models.disco.base:Using device:cuda:0
INFO:ekorpkit.models.disco.base:Disabling CUDNN for A100 gpu
INFO:ekorpkit.models.disco.base:> loading clip models...
Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
Loading model from: /opt/conda/lib/python3.8/site-packages/lpips/weights/v0.1/vgg.pth
text_prompts = "Dovish members of the Federal Reserve Board are to cut the target interest rates. matte style, artstation"
batch_name = "dovish"
# text_prompts = 'At a special meeting, hawkish central bankers are poised to raise the target rates. Trending on artstation, matte style'
# batch_name = "hawkish"
# text_prompts = "Mt. Halla's beautiful flowers, photorealistic"
# batch_name = "halla"
text_prompts = "Brave new world, matte style, artstaion"
batch_name = "newworld"
Image only mode#
disco.imagine(
text_prompts,
batch_name=batch_name,
n_samples=4,
show_collage=True,
width_height = [768, 512],
clip_guidance_scale=5000,
)
INFO:ekorpkit.models.disco.base:Seed used: 3001263848
INFO:ekorpkit.models.disco.base: >> elapsed time to diffuse: 0:11:33.275063
INFO:ekorpkit.models.art.base:Merging config with args: {}
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 1189633415
INFO:ekorpkit.models.art.base:Prompt: Brave new world, matte style, artstaion
4 samples generated to /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld
text prompts: {0: ['Brave new world, matte style, artstaion']}
sample image paths:
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld/newworld(1)_0000.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld/newworld(1)_0001.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld/newworld(1)_0002.png
/workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld/newworld(1)_0003.png
3D animation mode#
text_prompts = "Brave and beautiful new world in Jeju island, matte style, artstaion"
batch_name = "newworld-3d"
disco.imagine(
text_prompts,
batch_name=batch_name,
animation_mode = '3D',
max_frames = 100,
width_height = [768, 512],
clip_guidance_scale=5000,
)
INFO:ekorpkit.models.disco.base:translation: [-0.0, 0.0, -0.05]
INFO:ekorpkit.models.disco.base:rotation: [0.0, 0.0, 0.0]
INFO:ekorpkit.models.disco.base:rot_mat: tensor([[[1., 0., 0.],
[0., 1., 0.],
[0., 0., 1.]]], device='cuda:0')
translation_x: 0.0 translation_y: 0.0 translation_z: 10.0 rotation_3d_x: 0.0 rotation_3d_y: 0.0 rotation_3d_z: 0.0
Running AdaBins depth estimation implementation...
Loading base model ()...
Using cache found in /root/.cache/torch/hub/rwightman_gen-efficientnet-pytorch_master
Done.
Removing last two layers (global_pool & classifier).
Building Encoder-Decoder model..Done.
Running MiDaS depth estimation implementation...
Finished depth estimation.
INFO:ekorpkit.models.disco.base:Image prompt: []
INFO:ekorpkit.models.disco.base:Frame 99 Prompt: ['Brave and beautiful new world in Jeju island, matte style, artstaion']
INFO:ekorpkit.models.disco.base:Seed used: 1413882884
INFO:ekorpkit.models.disco.base: >> elapsed time to diffuse: 2:06:35.788987
Create a video from output flames#
batch_name = "newworld-3d"
batch_num = 0
mp4_path = disco.create_video(
batch_name=batch_name,
batch_num=batch_num,
)
INFO:ekorpkit.models.art.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld-3d/newworld-3d(0)_settings.yaml
INFO:ekorpkit.models.art.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Will save every 239 steps
INFO:ekorpkit.models.disco.base:Using seed: 187989656
INFO:ekorpkit.models.disco.base:Total frames: 100
The video is ready and saved to /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/newworld-3d/newworld-3d(0).mp4
from IPython.display import Video
Video(mp4_path, embed=True)
collage generated sample images#
disco.collage(
batch_name=batch_name,
batch_num=4,
ncols=2,
num_images=4,
show_filename=True,
fontcolor="black",
)
INFO:ekorpkit.models.dalle.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(4)_settings.yaml
INFO:ekorpkit.models.dalle.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 221812581
INFO:ekorpkit.models.dalle.base:Prompt: Mt. Halla's beautiful flowers, artstation matte painting
INFO:ekorpkit.io.file:Processing [4] files from ['halla(4)_*.png']
disco.collage(
batch_name=batch_name,
batch_num=1,
ncols=2,
num_images=4,
show_filename=True,
fontcolor="black",
)
INFO:ekorpkit.models.dalle.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/disco-diffusion/halla/halla(1)_settings.yaml
INFO:ekorpkit.models.dalle.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 1828352028
INFO:ekorpkit.models.dalle.base:Prompt: Mt. Halla's beautiful flowers
INFO:ekorpkit.io.file:Processing [4] files from ['halla(1)_*.png']
make gif of progresses of a specific sample image#
disco.make_gif(
batch_name='halla',
batch_num=1,
sample_num=1,
show=False,
duration=100,
force_remake=True,
)
Saved GIF to /workspace/projects/ekorpkit-book/disco-imagen/outputs/halla/halla(1)_0001.gif
show config#
disco.show_config(batch_name=batch_name, batch_num=4)
INFO:ekorpkit.models.disco.base:Loading config from /workspace/projects/ekorpkit-book/disco-imagen/outputs/halla/halla(4)_settings.yaml
INFO:ekorpkit.models.disco.base:Merging config with diffuse defaults
INFO:ekorpkit.models.disco.base:Merging config with args: {}
INFO:ekorpkit.models.disco.base:Will save every 2 steps
INFO:ekorpkit.models.disco.base:Using seed: 3370684779
{'angle': '0:(0)',
'animation_mode': 'None',
'batch_name': 'halla',
'batch_num': 7,
'batch_size': 1,
'calc_frames_skip_steps': 150,
'clamp_grad': True,
'clamp_max': 0.05,
'clip_denoised': False,
'clip_guidance_scale': 5000,
'cut_ic_pow': 1,
'cut_icgray_p': '[0.2]*400+[0]*600',
'cut_innercut': '[4]*400+[12]*600',
'cut_overview': '[12]*400+[4]*600',
'cutn_batches': 4,
'display_rate': 20,
'eta': 0.8,
'extract_nth_frame': 2,
'far_plane': 10000,
'force_flow_generation': False,
'fov': 40,
'frames_scale': 1500,
'frames_skip_steps': '60%',
'fuzzy_prompt': False,
'image_prompts': None,
'init_image': 'None',
'init_scale': 1000,
'intermediate_saves': 250,
'intermediates_in_subfolder': True,
'interp_spline': 'Linear',
'key_frames': True,
'max_frames': 1,
'midas_depth_model': 'dpt_large',
'midas_weight': 0.3,
'n_samples': 4,
'near_plane': 200,
'padding_mode': 'border',
'perlin_init': False,
'perlin_mode': 'mixed',
'persistent_frame_output_in_batch_folder': True,
'rand_mag': 0.05,
'randomize_class': True,
'range_scale': 150,
'resume_from_frame': 'latest',
'resume_run': False,
'retain_overwritten_frames': False,
'rotation_3d_x': '0: (0)',
'rotation_3d_y': '0: (0)',
'rotation_3d_z': '0: (0)',
'run_to_resume': 'latest',
'sampling_mode': 'bicubic',
'sat_scale': 0,
'seed': 3370684779,
'set_seed': 'random_seed',
'show_collage': True,
'side_x': 768,
'side_y': 512,
'skip_augs': False,
'skip_steps': 10,
'start_frame': 0,
'steps': 250,
'steps_per_checkpoint': 2,
'stop_on_next_loop': False,
'text_prompts': {0: ["Mt. Halla's beautiful flowers, artstation matte "
'painting'],
100: ['This set of prompts start at frame 100',
'This prompt has weight five:5']},
'transformation_percent': [0.09],
'translation_x': '0: (0)',
'translation_y': '0: (0)',
'translation_z': '0: (10.0)',
'turbo_mode': False,
'turbo_preroll': 10,
'turbo_steps': '3',
'tv_scale': 0,
'use_horizontal_symmetry': False,
'use_vertical_symmetry': False,
'video_init_blend_mode': 'optical flow',
'video_init_check_consistency': False,
'video_init_clip_guidance_scale': 1000,
'video_init_cutn_batches': 4,
'video_init_file': 'init.mp4',
'video_init_flow_blend': 0.999,
'video_init_flow_warp': True,
'video_init_frames_scale': 15000,
'video_init_frames_skip_steps': '70%',
'video_init_path': '/workspace/projects/ekorpkit-book/disco-imagen/init_images/init.mp4',
'video_init_range_scale': 150,
'video_init_sat_scale': 300,
'video_init_seed_continuity': False,
'video_init_skip_steps': 50,
'video_init_steps': 100,
'video_init_tv_scale': 0.1,
'vr_eye_angle': 0.5,
'vr_ipd': 5.0,
'vr_mode': False,
'width_height': [768, 512],
'zoom': '0: (1), 10: (1.05)'}